![]()
Get Backtest Orders¶
Get the backtest orders that you saved into the Object Store when you ran the benchmark and candidate algorithms in main.py.
In [ ]:
for kvp in qb.object_store:
key = kvp.key
value = kvp.value
print(key, value)
In [ ]:
qb = QuantBook()
def get_order_fills(key):
return pd.read_csv(
qb.object_store.get_file_path(f"{key}_order_fills"),
index_col=0, parse_dates=True
)
benchmark_orders = get_order_fills("benchmark")
candidate_orders = get_order_fills("candidate")
Calculate Summary Statistics¶
The following code block calculates some statistics and performs some sanity tests on the orders data.
In [6]:
same_index = candidate_orders.index == benchmark_orders.index
matching_fill_times = candidate_orders[same_index].index
different_fill_times_for_candidate = candidate_orders[~same_index].index
different_fill_times_for_benchmark = benchmark_orders[~same_index].index
print(f"Number of trades: {len(benchmark_orders)}")
print("Number of trades with the same fill times:", len(matching_fill_times))
print(
"Number of trades with different fill times:",
len(different_fill_times_for_candidate)
)
costs_saved = sum(
benchmark_orders['cost'].values - candidate_orders['cost'].values
)
print("Costs saved by delaying orders:", f"${round(costs_saved, 2)}")
# For all pairs of orders between the two algorithms, ensure the
# quantities match.
candidate_quantities = candidate_orders['quantity'].values
benchmark_quantities = benchmark_orders['quantity'].values
if not all(candidate_quantities == benchmark_quantities):
raise Exception('Error: The algorithms traded different quantities')
# For the orders that fill at the same time for both algorithms, ensure
# the fill price and costs also match.
candidate_info = candidate_orders.loc[matching_fill_times].drop('tag', axis=1)
benchmark_info = benchmark_orders.loc[matching_fill_times].drop('tag', axis=1)
if not all(candidate_info == benchmark_info):
raise Exception(
'Error: Fill prices and costs of identical orders do not match'
)
Number of trades: 366 Number of trades with the same fill times: 135 Number of trades with different fill times: 231 Costs saved by delaying orders: $6918.3
Analysis of Delayed Orders That Filled Before the Time Limit¶
Let's take a look at just the exit orders that were delayed but didn't hit the time limit and see how their costs compare to the costs of the benchmark orders.
In [7]:
from collections import Counter
import plotly.graph_objects as go
# Calculate number of orders that the algorithm delayed (and didn't hit
# the time limit).
index_numbers_of_time_limited_orders = []
index_numbers_of_delayed_orders = []
for i in range(len(benchmark_orders.index)):
# Skip orders that filled at the same time.
if benchmark_orders.index[i] == candidate_orders.index[i]:
continue
# Skip orders that hit the time limit.
if "Hit time limit" in candidate_orders.iloc[i]['tag']:
index_numbers_of_time_limited_orders.append(i)
continue
index_numbers_of_delayed_orders.append(i)
def display_order_results(title, index_numbers):
cost_deltas = (
benchmark_orders.iloc[index_numbers]['cost'].values
- candidate_orders.iloc[index_numbers]['cost'].values
)
candidate_dollar_volume = (
abs(candidate_orders.iloc[index_numbers]['quantity'])
* candidate_orders.iloc[index_numbers]['fill_price']
).values
cost_deltas_per_dollar_volume = cost_deltas / candidate_dollar_volume
fig = go.Figure(data=[go.Histogram(x=cost_deltas_per_dollar_volume, nbinsx=100)])
fig.update_layout(
title="Distribution of Costs Saved<br><sup>Costs saved for "
+ f"Delayed Orders that {title}</sup>",
xaxis_title="Costs Saved Per Dollar Volume (>0 => Candidate Saved Money)",
yaxis_title="Count"
)
fig.show()
print("Number of orders:", len(index_numbers))
pct_of_cheaper_orders = (
len(cost_deltas[cost_deltas > 0])
/ len(candidate_orders) * 100
)
print(
"Number of orders that lowered costs:",
len(cost_deltas[cost_deltas > 0]),
f"({round(pct_of_cheaper_orders, 2)}% of all orders)"
)
pct_of_no_change = (
len(cost_deltas[cost_deltas == 0])
/ len(candidate_orders) * 100
)
print(
"Number of orders that didn't change costs:",
len(cost_deltas[cost_deltas == 0]),
f"({round(pct_of_no_change, 2)}% of all orders)"
)
pct_of_raised_orders = (
len(cost_deltas[cost_deltas < 0])
/ len(candidate_orders) * 100
)
print(
"Number of orders that raised costs:",
len(cost_deltas[cost_deltas < 0]),
f"({round(pct_of_raised_orders, 2)}% of all orders)"
)
print(f"Total costs saved: ${round(sum(cost_deltas), 2)}")
display_order_results(
"Filled Before the Time Limit", index_numbers_of_delayed_orders
)
minutes_delayed = (
candidate_orders.iloc[index_numbers_of_delayed_orders].index
- benchmark_orders.iloc[index_numbers_of_delayed_orders].index
)
num_orders_by_delay_duration = pd.Series(
Counter([x.total_seconds() / 60 for x in minutes_delayed])
).sort_index()
fig = go.Figure(
go.Scatter(
x=num_orders_by_delay_duration.index,
y=num_orders_by_delay_duration.values, mode='lines'
)
)
fig.update_layout(
title="Number of Orders Filled Per Delay Duration<br><sup>Most order fill "
+ "within 90 minutes</sup>",
xaxis_title="Minutes Delayed",
yaxis_title="Number of Orders Filled"
)
fig.show()
Number of orders: 219 Number of orders that lowered costs: 218 (59.56% of all orders) Number of orders that didn't change costs: 0 (0.0% of all orders) Number of orders that raised costs: 1 (0.27% of all orders) Total costs saved: $6852.0
Analysis of Delayed Orders That Hit the Time Limit¶
Now let's take a look at just the exit orders that hit the time limit and how their costs compare to the costs of the benchmark orders.
In [8]:
display_order_results(
"Hit the Time Limit", index_numbers_of_time_limited_orders
)
Number of orders: 12 Number of orders that lowered costs: 4 (1.09% of all orders) Number of orders that didn't change costs: 4 (1.09% of all orders) Number of orders that raised costs: 4 (1.09% of all orders) Total costs saved: $66.3